In [ ]:
from google.colab import drive
drive.mount('/content/drive')
Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount("/content/drive", force_remount=True).
In [ ]:
train_dir = '/content/drive/MyDrive/Lalith/cv_dataset/train'
val_dir = '/content/drive/MyDrive/Lalith/cv_dataset/val'
test_dir = '/content/drive/MyDrive/Lalith/cv_dataset/test'
In [ ]:
from tensorflow.keras.preprocessing.image import ImageDataGenerator
# Initialize the data generators with rescaling
datagen = ImageDataGenerator(rescale=1./255)
# Set up data generators for training, validation, and testing
train_generator = datagen.flow_from_directory(
train_dir, target_size=(224, 224), batch_size=32, class_mode='input'
)
val_generator = datagen.flow_from_directory(
val_dir, target_size=(224, 224), batch_size=32, class_mode='input'
)
test_generator = datagen.flow_from_directory(
test_dir, target_size=(224, 224), batch_size=32, class_mode='input'
)
Found 465 images belonging to 1 classes. Found 41 images belonging to 1 classes. Found 10 images belonging to 1 classes.
In [ ]:
import matplotlib.pyplot as plt
# Display a batch of images from the training set
batch = next(train_generator)
plt.imshow(batch[0][0]) # Show the first image in the batch
plt.axis('off')
plt.show()
In [ ]:
from tensorflow.keras import layers, models
def create_sr_model(input_shape=(224, 224, 3)):
inputs = layers.Input(shape=input_shape)
# Basic super-resolution CNN layers
x = layers.Conv2D(64, (3, 3), activation='relu', padding='same')(inputs)
x = layers.Conv2D(64, (3, 3), activation='relu', padding='same')(x)
x = layers.Conv2D(64, (3, 3), activation='relu', padding='same')(x)
x = layers.Conv2D(3, (3, 3), activation='sigmoid', padding='same')(x) # Output layer
model = models.Model(inputs, x)
return model
# Create the model instance
sr_model = create_sr_model()
sr_model.summary()
Model: "functional_3"
┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓ ┃ Layer (type) ┃ Output Shape ┃ Param # ┃ ┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩ │ input_layer_3 (InputLayer) │ (None, 224, 224, 3) │ 0 │ ├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤ │ conv2d_4 (Conv2D) │ (None, 224, 224, 64) │ 1,792 │ ├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤ │ conv2d_5 (Conv2D) │ (None, 224, 224, 64) │ 36,928 │ ├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤ │ conv2d_6 (Conv2D) │ (None, 224, 224, 64) │ 36,928 │ ├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤ │ conv2d_7 (Conv2D) │ (None, 224, 224, 3) │ 1,731 │ └──────────────────────────────────────┴─────────────────────────────┴─────────────────┘
Total params: 77,379 (302.26 KB)
Trainable params: 77,379 (302.26 KB)
Non-trainable params: 0 (0.00 B)
In [ ]:
from tensorflow.keras.applications import VGG19
from tensorflow.keras.models import Model
# Load VGG19 without the top layer, pretrained on ImageNet
vgg = VGG19(weights='imagenet', include_top=False, input_shape=(224, 224, 3))
# Select a specific layer for perceptual loss (e.g., 'block4_conv4')
vgg_model = Model(inputs=vgg.input, outputs=vgg.get_layer('block4_conv4').output)
vgg_model.trainable = False # Freeze VGG19
In [ ]:
def perceptual_loss(y_true, y_pred):
y_pred_resized = tf.image.resize(y_pred, (224, 224))
y_true_resized = tf.image.resize(y_true, (224, 224))
y_true_features = vgg_model(y_true_resized)
y_pred_features = vgg_model(y_pred_resized)
return tf.reduce_mean(tf.square(y_true_features - y_pred_features))
In [ ]:
sr_model.compile(optimizer='adam', loss=perceptual_loss, metrics=['mse'])
In [ ]:
batch_size = 32
steps_per_epoch = len(train_generator) # Calculate based on generator size
validation_steps = len(val_generator) # Calculate based on generator size
In [ ]:
from tensorflow.keras.applications import VGG19
from tensorflow.keras.models import Model
import tensorflow as tf
# Load VGG19 for perceptual loss
vgg = VGG19(weights='imagenet', include_top=False, input_shape=(224, 224, 3))
vgg_model = Model(inputs=vgg.input, outputs=vgg.get_layer('block4_conv4').output)
vgg_model.trainable = False
# Define perceptual loss
def perceptual_loss(y_true, y_pred):
y_true_features = vgg_model(y_true)
y_pred_features = vgg_model(y_pred)
return tf.reduce_mean(tf.square(y_true_features - y_pred_features))
# Compile sr_model with perceptual loss
sr_model.compile(optimizer='adam', loss=perceptual_loss, metrics=['mse'])
In [ ]:
batch = next(train_generator)
print(batch[0].shape) # Should print something like (32, 224, 224, 3) for batch size 32
(32, 224, 224, 3)
In [ ]:
from PIL import Image
import os
# Convert images to RGB if they have transparency issues and log changes
for folder in [train_dir, val_dir, test_dir]:
for root, _, files in os.walk(folder):
for filename in files:
img_path = os.path.join(root, filename)
try:
with Image.open(img_path) as img:
if img.mode != 'RGB':
img = img.convert("RGB")
img.save(img_path)
print(f"Converted {img_path} to RGB")
except Exception as e:
print(f"Error processing {img_path}: {e}")
In [ ]:
history = sr_model.fit(
train_generator,
validation_data=val_generator,
epochs=50
)
Epoch 1/50 15/15 ━━━━━━━━━━━━━━━━━━━━ 20s 977ms/step - loss: 6.6576 - mse: 0.1108 - val_loss: 2.8609 - val_mse: 0.1549 Epoch 2/50 15/15 ━━━━━━━━━━━━━━━━━━━━ 17s 813ms/step - loss: 2.9012 - mse: 0.1899 - val_loss: 1.9351 - val_mse: 0.1729 Epoch 3/50 15/15 ━━━━━━━━━━━━━━━━━━━━ 15s 838ms/step - loss: 2.1879 - mse: 0.1780 - val_loss: 1.6879 - val_mse: 0.1382 Epoch 4/50 15/15 ━━━━━━━━━━━━━━━━━━━━ 21s 846ms/step - loss: 1.8455 - mse: 0.1273 - val_loss: 1.4775 - val_mse: 0.0873 Epoch 5/50 15/15 ━━━━━━━━━━━━━━━━━━━━ 14s 818ms/step - loss: 1.6514 - mse: 0.0859 - val_loss: 1.1323 - val_mse: 0.0393 Epoch 6/50 15/15 ━━━━━━━━━━━━━━━━━━━━ 14s 840ms/step - loss: 1.2054 - mse: 0.0355 - val_loss: 0.7639 - val_mse: 0.0167 Epoch 7/50 15/15 ━━━━━━━━━━━━━━━━━━━━ 14s 816ms/step - loss: 0.7843 - mse: 0.0192 - val_loss: 0.5179 - val_mse: 0.0145 Epoch 8/50 15/15 ━━━━━━━━━━━━━━━━━━━━ 20s 803ms/step - loss: 0.5856 - mse: 0.0185 - val_loss: 0.4312 - val_mse: 0.0126 Epoch 9/50 15/15 ━━━━━━━━━━━━━━━━━━━━ 14s 826ms/step - loss: 0.4799 - mse: 0.0150 - val_loss: 0.3570 - val_mse: 0.0100 Epoch 10/50 15/15 ━━━━━━━━━━━━━━━━━━━━ 14s 819ms/step - loss: 0.4053 - mse: 0.0127 - val_loss: 0.3181 - val_mse: 0.0079 Epoch 11/50 15/15 ━━━━━━━━━━━━━━━━━━━━ 14s 814ms/step - loss: 0.3422 - mse: 0.0108 - val_loss: 0.2590 - val_mse: 0.0061 Epoch 12/50 15/15 ━━━━━━━━━━━━━━━━━━━━ 21s 833ms/step - loss: 0.2805 - mse: 0.0090 - val_loss: 0.2261 - val_mse: 0.0050 Epoch 13/50 15/15 ━━━━━━━━━━━━━━━━━━━━ 20s 815ms/step - loss: 0.2394 - mse: 0.0073 - val_loss: 0.1734 - val_mse: 0.0042 Epoch 14/50 15/15 ━━━━━━━━━━━━━━━━━━━━ 21s 833ms/step - loss: 0.1872 - mse: 0.0059 - val_loss: 0.1445 - val_mse: 0.0030 Epoch 15/50 15/15 ━━━━━━━━━━━━━━━━━━━━ 20s 816ms/step - loss: 0.1545 - mse: 0.0047 - val_loss: 0.1130 - val_mse: 0.0032 Epoch 16/50 15/15 ━━━━━━━━━━━━━━━━━━━━ 14s 820ms/step - loss: 0.1267 - mse: 0.0040 - val_loss: 0.0918 - val_mse: 0.0029 Epoch 17/50 15/15 ━━━━━━━━━━━━━━━━━━━━ 20s 803ms/step - loss: 0.0970 - mse: 0.0034 - val_loss: 0.0825 - val_mse: 0.0033 Epoch 18/50 15/15 ━━━━━━━━━━━━━━━━━━━━ 21s 822ms/step - loss: 0.0876 - mse: 0.0034 - val_loss: 0.0737 - val_mse: 0.0039 Epoch 19/50 15/15 ━━━━━━━━━━━━━━━━━━━━ 14s 839ms/step - loss: 0.0791 - mse: 0.0033 - val_loss: 0.0698 - val_mse: 0.0042 Epoch 20/50 15/15 ━━━━━━━━━━━━━━━━━━━━ 20s 811ms/step - loss: 0.0700 - mse: 0.0033 - val_loss: 0.0572 - val_mse: 0.0015 Epoch 21/50 15/15 ━━━━━━━━━━━━━━━━━━━━ 21s 847ms/step - loss: 0.0606 - mse: 0.0025 - val_loss: 0.0522 - val_mse: 0.0019 Epoch 22/50 15/15 ━━━━━━━━━━━━━━━━━━━━ 15s 863ms/step - loss: 0.0597 - mse: 0.0026 - val_loss: 0.0482 - val_mse: 0.0022 Epoch 23/50 15/15 ━━━━━━━━━━━━━━━━━━━━ 20s 815ms/step - loss: 0.0516 - mse: 0.0023 - val_loss: 0.0451 - val_mse: 0.0025 Epoch 24/50 15/15 ━━━━━━━━━━━━━━━━━━━━ 21s 851ms/step - loss: 0.0462 - mse: 0.0023 - val_loss: 0.0429 - val_mse: 0.0019 Epoch 25/50 15/15 ━━━━━━━━━━━━━━━━━━━━ 20s 817ms/step - loss: 0.0457 - mse: 0.0022 - val_loss: 0.0429 - val_mse: 0.0015 Epoch 26/50 15/15 ━━━━━━━━━━━━━━━━━━━━ 14s 833ms/step - loss: 0.0463 - mse: 0.0021 - val_loss: 0.0395 - val_mse: 0.0026 Epoch 27/50 15/15 ━━━━━━━━━━━━━━━━━━━━ 14s 815ms/step - loss: 0.0420 - mse: 0.0021 - val_loss: 0.0350 - val_mse: 0.0013 Epoch 28/50 15/15 ━━━━━━━━━━━━━━━━━━━━ 20s 792ms/step - loss: 0.0380 - mse: 0.0018 - val_loss: 0.0349 - val_mse: 0.0012 Epoch 29/50 15/15 ━━━━━━━━━━━━━━━━━━━━ 14s 813ms/step - loss: 0.0365 - mse: 0.0019 - val_loss: 0.0332 - val_mse: 0.0017 Epoch 30/50 15/15 ━━━━━━━━━━━━━━━━━━━━ 14s 826ms/step - loss: 0.0401 - mse: 0.0019 - val_loss: 0.0346 - val_mse: 0.0011 Epoch 31/50 15/15 ━━━━━━━━━━━━━━━━━━━━ 20s 811ms/step - loss: 0.0423 - mse: 0.0019 - val_loss: 0.0342 - val_mse: 0.0014 Epoch 32/50 15/15 ━━━━━━━━━━━━━━━━━━━━ 20s 813ms/step - loss: 0.0348 - mse: 0.0017 - val_loss: 0.0323 - val_mse: 0.0018 Epoch 33/50 15/15 ━━━━━━━━━━━━━━━━━━━━ 21s 830ms/step - loss: 0.0326 - mse: 0.0019 - val_loss: 0.0269 - val_mse: 0.0014 Epoch 34/50 15/15 ━━━━━━━━━━━━━━━━━━━━ 20s 812ms/step - loss: 0.0304 - mse: 0.0016 - val_loss: 0.0317 - val_mse: 0.0026 Epoch 35/50 15/15 ━━━━━━━━━━━━━━━━━━━━ 14s 840ms/step - loss: 0.0317 - mse: 0.0019 - val_loss: 0.0281 - val_mse: 0.0023 Epoch 36/50 15/15 ━━━━━━━━━━━━━━━━━━━━ 14s 814ms/step - loss: 0.0308 - mse: 0.0020 - val_loss: 0.0251 - val_mse: 9.9029e-04 Epoch 37/50 15/15 ━━━━━━━━━━━━━━━━━━━━ 14s 809ms/step - loss: 0.0290 - mse: 0.0017 - val_loss: 0.0267 - val_mse: 8.1171e-04 Epoch 38/50 15/15 ━━━━━━━━━━━━━━━━━━━━ 14s 806ms/step - loss: 0.0264 - mse: 0.0013 - val_loss: 0.0241 - val_mse: 0.0015 Epoch 39/50 15/15 ━━━━━━━━━━━━━━━━━━━━ 14s 806ms/step - loss: 0.0265 - mse: 0.0015 - val_loss: 0.0254 - val_mse: 0.0019 Epoch 40/50 15/15 ━━━━━━━━━━━━━━━━━━━━ 14s 851ms/step - loss: 0.0257 - mse: 0.0016 - val_loss: 0.0242 - val_mse: 0.0016 Epoch 41/50 15/15 ━━━━━━━━━━━━━━━━━━━━ 20s 804ms/step - loss: 0.0253 - mse: 0.0015 - val_loss: 0.0254 - val_mse: 0.0021 Epoch 42/50 15/15 ━━━━━━━━━━━━━━━━━━━━ 20s 823ms/step - loss: 0.0258 - mse: 0.0017 - val_loss: 0.0225 - val_mse: 0.0010 Epoch 43/50 15/15 ━━━━━━━━━━━━━━━━━━━━ 21s 819ms/step - loss: 0.0262 - mse: 0.0014 - val_loss: 0.0211 - val_mse: 8.6176e-04 Epoch 44/50 15/15 ━━━━━━━━━━━━━━━━━━━━ 14s 828ms/step - loss: 0.0241 - mse: 0.0013 - val_loss: 0.0218 - val_mse: 7.0237e-04 Epoch 45/50 15/15 ━━━━━━━━━━━━━━━━━━━━ 20s 803ms/step - loss: 0.0259 - mse: 0.0014 - val_loss: 0.0227 - val_mse: 7.1011e-04 Epoch 46/50 15/15 ━━━━━━━━━━━━━━━━━━━━ 21s 806ms/step - loss: 0.0241 - mse: 0.0013 - val_loss: 0.0208 - val_mse: 7.0958e-04 Epoch 47/50 15/15 ━━━━━━━━━━━━━━━━━━━━ 21s 823ms/step - loss: 0.0221 - mse: 0.0013 - val_loss: 0.0199 - val_mse: 0.0014 Epoch 48/50 15/15 ━━━━━━━━━━━━━━━━━━━━ 14s 830ms/step - loss: 0.0206 - mse: 0.0014 - val_loss: 0.0190 - val_mse: 0.0010 Epoch 49/50 15/15 ━━━━━━━━━━━━━━━━━━━━ 14s 815ms/step - loss: 0.0204 - mse: 0.0013 - val_loss: 0.0201 - val_mse: 0.0017 Epoch 50/50 15/15 ━━━━━━━━━━━━━━━━━━━━ 14s 804ms/step - loss: 0.0213 - mse: 0.0014 - val_loss: 0.0256 - val_mse: 0.0025
In [ ]:
import matplotlib.pyplot as plt
# Plot the training and validation loss
plt.plot(history.history['loss'], label='Training Loss')
plt.plot(history.history['val_loss'], label='Validation Loss')
plt.xlabel('Epoch')
plt.ylabel('Loss')
plt.legend()
plt.show()
# Plot the training and validation MSE
plt.plot(history.history['mse'], label='Training MSE')
plt.plot(history.history['val_mse'], label='Validation MSE')
plt.xlabel('Epoch')
plt.ylabel('MSE')
plt.legend()
plt.show()
In [ ]:
test_loss, test_mse = sr_model.evaluate(test_generator, steps=len(test_generator))
print(f"Test Loss: {test_loss}, Test MSE: {test_mse}")
1/1 ━━━━━━━━━━━━━━━━━━━━ 9s 9s/step - loss: 0.0288 - mse: 0.0031 Test Loss: 0.028842534869909286, Test MSE: 0.003077416680753231
In [ ]:
import numpy as np
# Get a batch of test images
test_batch = next(test_generator)
low_res_images = test_batch[0] # Low-resolution inputs
# Predict high-resolution images
sr_images = sr_model.predict(low_res_images)
# Display low-resolution and super-resolution images side by side
for i in range(5): # Display first 5 images in the batch
plt.figure(figsize=(10, 5))
# Low-resolution image
plt.subplot(1, 2, 1)
plt.title("Low-Resolution")
plt.imshow(np.clip(low_res_images[i], 0, 1))
plt.axis('off')
# Super-resolution image
plt.subplot(1, 2, 2)
plt.title("Super-Resolution")
plt.imshow(np.clip(sr_images[i], 0, 1))
plt.axis('off')
plt.show()
1/1 ━━━━━━━━━━━━━━━━━━━━ 0s 323ms/step
In [33]:
sr_model.save('/content/drive/MyDrive/super_resolution_model.h5')
WARNING:absl:You are saving your model as an HDF5 file via `model.save()` or `keras.saving.save_model(model)`. This file format is considered legacy. We recommend using instead the native Keras format, e.g. `model.save('my_model.keras')` or `keras.saving.save_model(model, 'my_model.keras')`.
In [35]:
#peak signal to noise ratio
import tensorflow as tf
def calculate_psnr(y_true, y_pred):
return tf.image.psnr(y_true, y_pred, max_val=1.0) # Assume pixel values are normalized [0,1]
# Example on a batch of test images
test_batch = next(test_generator)
low_res_images, high_res_images = test_batch # Get low-res and high-res images
sr_images = sr_model.predict(low_res_images) # Get super-res images
# Calculate PSNR for each image in the batch and average them
psnr_values = [calculate_psnr(high_res_images[i], sr_images[i]) for i in range(len(sr_images))]
avg_psnr = tf.reduce_mean(psnr_values)
print(f"Average PSNR on test set: {avg_psnr:.2f} dB")
1/1 ━━━━━━━━━━━━━━━━━━━━ 0s 22ms/step Average PSNR on test set: 25.44 dB
In [36]:
#structural similarity index
def calculate_ssim(y_true, y_pred):
return tf.image.ssim(y_true, y_pred, max_val=1.0) # Assume pixel values are normalized [0,1]
# Calculate SSIM for each image in the batch and average them
ssim_values = [calculate_ssim(high_res_images[i], sr_images[i]) for i in range(len(sr_images))]
avg_ssim = tf.reduce_mean(ssim_values)
print(f"Average SSIM on test set: {avg_ssim:.4f}")
Average SSIM on test set: 0.9687
Perceptual Loss
In [37]:
from tensorflow.keras.applications import VGG19
from tensorflow.keras.models import Model
# Load VGG19 and set it up for feature extraction
vgg = VGG19(weights='imagenet', include_top=False, input_shape=(224, 224, 3))
# Choose an intermediate layer for perceptual similarity (e.g., 'block4_conv4')
feature_layer = 'block4_conv4'
vgg_model = Model(inputs=vgg.input, outputs=vgg.get_layer(feature_layer).output)
vgg_model.trainable = False # Freeze the VGG model
In [38]:
import tensorflow as tf
def perceptual_loss(y_true, y_pred):
# Extract features of the true and predicted images
y_true_features = vgg_model(y_true)
y_pred_features = vgg_model(y_pred)
# Calculate the mean squared error between the features
return tf.reduce_mean(tf.square(y_true_features - y_pred_features))
In [39]:
# Get a batch of test images
test_batch = next(test_generator)
low_res_images, high_res_images = test_batch # Low-res inputs and high-res targets
# Generate super-resolution images
sr_images = sr_model.predict(low_res_images)
# Calculate perceptual loss for each image pair in the batch
perceptual_losses = [perceptual_loss(tf.expand_dims(high_res_images[i], 0),
tf.expand_dims(sr_images[i], 0)) for i in range(len(sr_images))]
# Average perceptual loss over the batch
avg_perceptual_loss = tf.reduce_mean(perceptual_losses)
print(f"Average Perceptual Loss on test set: {avg_perceptual_loss:.4f}")
1/1 ━━━━━━━━━━━━━━━━━━━━ 0s 31ms/step Average Perceptual Loss on test set: 0.0288
In [40]:
import numpy as np
import matplotlib.pyplot as plt
# Get a batch of test images
test_batch = next(test_generator)
low_res_images, high_res_images = test_batch # Low-res inputs and high-res targets
sr_images = sr_model.predict(low_res_images) # Generate super-resolved images
# Display a few examples
num_examples = 5 # Number of examples to display
for i in range(num_examples):
plt.figure(figsize=(12, 4))
# Low-resolution input
plt.subplot(1, 3, 1)
plt.title("Low-Resolution")
plt.imshow(np.clip(low_res_images[i], 0, 1))
plt.axis('off')
# Super-resolution output
plt.subplot(1, 3, 2)
plt.title("Super-Resolution")
plt.imshow(np.clip(sr_images[i], 0, 1))
plt.axis('off')
# Ground-truth high-resolution
plt.subplot(1, 3, 3)
plt.title("High-Resolution (Ground Truth)")
plt.imshow(np.clip(high_res_images[i], 0, 1))
plt.axis('off')
plt.show()
1/1 ━━━━━━━━━━━━━━━━━━━━ 0s 23ms/step
In [41]:
import tensorflow as tf
def calculate_psnr(y_true, y_pred):
return tf.image.psnr(y_true, y_pred, max_val=1.0) # Assuming pixel values are in [0, 1]
# Calculate PSNR for each image in the test batch and get the average
psnr_values = [calculate_psnr(high_res_images[i], sr_images[i]) for i in range(len(sr_images))]
avg_psnr = tf.reduce_mean(psnr_values)
print(f"Average PSNR on test set: {avg_psnr:.2f} dB")
Average PSNR on test set: 25.44 dB
In [42]:
def calculate_ssim(y_true, y_pred):
return tf.image.ssim(y_true, y_pred, max_val=1.0) # Assuming pixel values are in [0, 1]
# Calculate SSIM for each image in the test batch and get the average
ssim_values = [calculate_ssim(high_res_images[i], sr_images[i]) for i in range(len(sr_images))]
avg_ssim = tf.reduce_mean(ssim_values)
print(f"Average SSIM on test set: {avg_ssim:.4f}")
Average SSIM on test set: 0.9687